{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.157487Z",
     "start_time": "2024-03-04T09:28:52.154486Z"
    }
   },
   "outputs": [],
   "source": [
    "from scipy.io import loadmat\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "{'__header__': b'MATLAB 5.0 MAT-file, Platform: GLNXA64, Created on: Tue Oct 18 14:57:02 2011',\n '__version__': '1.0',\n '__globals__': [],\n 'Theta1': array([[-2.25623899e-02, -1.05624163e-08,  2.19414684e-09, ...,\n         -1.30529929e-05, -5.04175101e-06,  2.80464449e-09],\n        [-9.83811294e-02,  7.66168682e-09, -9.75873689e-09, ...,\n         -5.60134007e-05,  2.00940969e-07,  3.54422854e-09],\n        [ 1.16156052e-01, -8.77654466e-09,  8.16037764e-09, ...,\n         -1.20951657e-04, -2.33669661e-06, -7.50668099e-09],\n        ...,\n        [-1.83220638e-01, -8.89272060e-09, -9.81968100e-09, ...,\n          2.35311186e-05, -3.25484493e-06,  9.02499060e-09],\n        [-7.02096331e-01,  3.05178374e-10,  2.56061008e-09, ...,\n         -8.61759744e-04,  9.43449909e-05,  3.83761998e-09],\n        [-3.50933229e-01,  8.85876862e-09, -6.57515140e-10, ...,\n         -1.80365926e-06, -8.14464807e-06,  8.79454531e-09]]),\n 'Theta2': array([[-0.76100352, -1.21244498, -0.10187131, -2.36850085, -1.05778129,\n         -2.20823629,  0.56383834,  1.21105294,  2.21030997,  0.44456156,\n         -1.18244872,  1.04289112, -1.60558756,  1.30419943,  1.37175046,\n          1.74825095, -0.23365648, -1.52014483,  1.15324176,  0.10368082,\n         -0.37207719, -0.61530019, -0.1256836 , -2.27193038, -0.71836208,\n         -1.29690315],\n        [-0.61785176,  0.61559207, -1.26550639,  1.85745418, -0.91853319,\n         -0.05502589, -0.38589806,  1.29520853, -1.56843297, -0.97026419,\n         -2.18334895, -2.85033578, -2.07733086,  1.63163164,  0.3490229 ,\n          1.82789117, -2.44174379, -0.8563034 , -0.2982564 , -2.07947873,\n         -1.2933238 ,  0.89982032,  0.28306578,  2.31180525, -2.46444086,\n          1.45656548],\n        [-0.68934072, -1.94538151,  2.01360618, -3.12316188, -0.2361763 ,\n          1.38680947,  0.90982429, -1.54774416, -0.79830896, -0.65599834,\n          0.7353833 , -2.58593294,  0.47210839,  0.55349499,  2.51255453,\n         -2.4167454 , -1.63898627,  1.2027302 , -1.20245851, -1.83445959,\n         -1.88013027, -0.34056098,  0.23692483, -1.06137919,  1.02759232,\n         -0.47690832],\n        [-0.67832479,  0.46299226,  0.58492321, -0.1650184 ,  1.93264192,\n         -0.22965765, -1.84731492,  0.49011768,  1.07146054, -3.31905643,\n          1.54113507,  0.37371947, -0.86484681, -2.58273522,  0.97062447,\n         -0.51021867, -0.68427897, -1.64713607,  0.21153145, -0.27422442,\n          1.72599755,  1.32418658, -2.63984479, -0.08055871, -2.03510803,\n         -1.46123776],\n        [-0.59664339, -2.04481799,  2.05698407,  1.95100909,  0.17637699,\n         -2.16141218, -0.40394736,  1.80157532, -1.56278739, -0.25253004,\n          0.23586497,  0.71656699,  1.07689092, -0.35457279, -1.67743058,\n         -0.12939255, -0.67488849,  1.14066535,  1.32431237,  3.21158484,\n         -2.15888898, -2.60164082, -3.2226466 , -1.89612906, -0.87488068,\n          2.51038628],\n        [-0.87794907,  0.4344112 , -0.93161049,  0.18390778, -0.36078216,\n          0.61958137,  0.38624948, -2.65150343,  2.29710773, -2.08818098,\n         -1.86382323,  1.06057836,  0.77562146,  2.1346861 , -1.14973702,\n         -0.52081426,  0.99743429, -1.48309353, -2.3139424 ,  0.29517333,\n         -0.38704879, -2.20607697,  0.30702191, -1.17646114, -1.63462966,\n         -0.82467661],\n        [-0.52746527,  1.21564288, -1.50095981, -2.03195359, -1.52366734,\n         -2.43732079, -2.37570311, -1.39987277, -0.88735315, -0.63278873,\n          1.50450176, -1.580763  ,  0.58599217, -0.77540416,  0.94257331,\n          2.10919653,  0.54479132,  0.43773612, -1.28024228, -0.04360994,\n          1.4774997 , -1.13276949, -0.72846904,  0.04734716,  1.6574566 ,\n          1.68540944],\n        [-0.7490154 , -0.72249056, -3.15228173,  0.36577778,  0.19811362,\n         -0.73059946,  1.65263918, -2.300357  , -1.87468162,  0.98095387,\n         -1.58825159,  1.35434142,  2.17895331, -1.99239762, -2.00371362,\n         -0.388613  , -2.33992976, -2.91719062,  0.99398645, -2.70476768,\n         -1.27139772,  1.86091461, -1.20519404, -0.38014194,  0.7087181 ,\n         -2.11014003],\n        [-0.6665468 ,  0.53601845,  1.30307573, -1.03372714, -4.03084753,\n          0.58173469, -2.65717902,  0.80379994, -1.09241928,  2.49910058,\n          0.362008  ,  0.66195337, -0.92160534, -0.83123666, -2.00200952,\n         -2.94897501,  0.64564202, -1.10114694,  0.74510309,  0.58506717,\n         -1.99545251,  0.62591105,  1.80596103, -0.22309315, -1.40442136,\n         -2.1319153 ],\n        [-0.46089119, -1.43944954, -1.21809509,  0.71093011,  0.45216919,\n         -0.35953381,  0.62284954, -0.67005297, -0.7069138 ,  0.06311351,\n         -1.23199074, -1.74645233, -2.71960897, -2.21437178, -1.69307505,\n         -0.90927394,  0.87852311,  1.18664814, -1.87041262,  0.39796295,\n          1.72113872, -1.36934055,  0.8580668 , -0.24779579,  1.28009118,\n         -1.32752042]])}"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "weights = loadmat('ex3weights.mat')\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.166838Z",
     "start_time": "2024-03-04T09:28:52.158487Z"
    }
   },
   "id": "f622f1d42c095fd6",
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "array([[-2.25623899e-02, -1.05624163e-08,  2.19414684e-09, ...,\n        -1.30529929e-05, -5.04175101e-06,  2.80464449e-09],\n       [-9.83811294e-02,  7.66168682e-09, -9.75873689e-09, ...,\n        -5.60134007e-05,  2.00940969e-07,  3.54422854e-09],\n       [ 1.16156052e-01, -8.77654466e-09,  8.16037764e-09, ...,\n        -1.20951657e-04, -2.33669661e-06, -7.50668099e-09],\n       ...,\n       [-1.83220638e-01, -8.89272060e-09, -9.81968100e-09, ...,\n         2.35311186e-05, -3.25484493e-06,  9.02499060e-09],\n       [-7.02096331e-01,  3.05178374e-10,  2.56061008e-09, ...,\n        -8.61759744e-04,  9.43449909e-05,  3.83761998e-09],\n       [-3.50933229e-01,  8.85876862e-09, -6.57515140e-10, ...,\n        -1.80365926e-06, -8.14464807e-06,  8.79454531e-09]])"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta1 = np.array(weights[\"Theta1\"], dtype=\"float64\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.171088Z",
     "start_time": "2024-03-04T09:28:52.167350Z"
    }
   },
   "id": "b26ccfc94fd639b6",
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "array([[-0.76100352, -1.21244498, -0.10187131, -2.36850085, -1.05778129,\n        -2.20823629,  0.56383834,  1.21105294,  2.21030997,  0.44456156,\n        -1.18244872,  1.04289112, -1.60558756,  1.30419943,  1.37175046,\n         1.74825095, -0.23365648, -1.52014483,  1.15324176,  0.10368082,\n        -0.37207719, -0.61530019, -0.1256836 , -2.27193038, -0.71836208,\n        -1.29690315],\n       [-0.61785176,  0.61559207, -1.26550639,  1.85745418, -0.91853319,\n        -0.05502589, -0.38589806,  1.29520853, -1.56843297, -0.97026419,\n        -2.18334895, -2.85033578, -2.07733086,  1.63163164,  0.3490229 ,\n         1.82789117, -2.44174379, -0.8563034 , -0.2982564 , -2.07947873,\n        -1.2933238 ,  0.89982032,  0.28306578,  2.31180525, -2.46444086,\n         1.45656548],\n       [-0.68934072, -1.94538151,  2.01360618, -3.12316188, -0.2361763 ,\n         1.38680947,  0.90982429, -1.54774416, -0.79830896, -0.65599834,\n         0.7353833 , -2.58593294,  0.47210839,  0.55349499,  2.51255453,\n        -2.4167454 , -1.63898627,  1.2027302 , -1.20245851, -1.83445959,\n        -1.88013027, -0.34056098,  0.23692483, -1.06137919,  1.02759232,\n        -0.47690832],\n       [-0.67832479,  0.46299226,  0.58492321, -0.1650184 ,  1.93264192,\n        -0.22965765, -1.84731492,  0.49011768,  1.07146054, -3.31905643,\n         1.54113507,  0.37371947, -0.86484681, -2.58273522,  0.97062447,\n        -0.51021867, -0.68427897, -1.64713607,  0.21153145, -0.27422442,\n         1.72599755,  1.32418658, -2.63984479, -0.08055871, -2.03510803,\n        -1.46123776],\n       [-0.59664339, -2.04481799,  2.05698407,  1.95100909,  0.17637699,\n        -2.16141218, -0.40394736,  1.80157532, -1.56278739, -0.25253004,\n         0.23586497,  0.71656699,  1.07689092, -0.35457279, -1.67743058,\n        -0.12939255, -0.67488849,  1.14066535,  1.32431237,  3.21158484,\n        -2.15888898, -2.60164082, -3.2226466 , -1.89612906, -0.87488068,\n         2.51038628],\n       [-0.87794907,  0.4344112 , -0.93161049,  0.18390778, -0.36078216,\n         0.61958137,  0.38624948, -2.65150343,  2.29710773, -2.08818098,\n        -1.86382323,  1.06057836,  0.77562146,  2.1346861 , -1.14973702,\n        -0.52081426,  0.99743429, -1.48309353, -2.3139424 ,  0.29517333,\n        -0.38704879, -2.20607697,  0.30702191, -1.17646114, -1.63462966,\n        -0.82467661],\n       [-0.52746527,  1.21564288, -1.50095981, -2.03195359, -1.52366734,\n        -2.43732079, -2.37570311, -1.39987277, -0.88735315, -0.63278873,\n         1.50450176, -1.580763  ,  0.58599217, -0.77540416,  0.94257331,\n         2.10919653,  0.54479132,  0.43773612, -1.28024228, -0.04360994,\n         1.4774997 , -1.13276949, -0.72846904,  0.04734716,  1.6574566 ,\n         1.68540944],\n       [-0.7490154 , -0.72249056, -3.15228173,  0.36577778,  0.19811362,\n        -0.73059946,  1.65263918, -2.300357  , -1.87468162,  0.98095387,\n        -1.58825159,  1.35434142,  2.17895331, -1.99239762, -2.00371362,\n        -0.388613  , -2.33992976, -2.91719062,  0.99398645, -2.70476768,\n        -1.27139772,  1.86091461, -1.20519404, -0.38014194,  0.7087181 ,\n        -2.11014003],\n       [-0.6665468 ,  0.53601845,  1.30307573, -1.03372714, -4.03084753,\n         0.58173469, -2.65717902,  0.80379994, -1.09241928,  2.49910058,\n         0.362008  ,  0.66195337, -0.92160534, -0.83123666, -2.00200952,\n        -2.94897501,  0.64564202, -1.10114694,  0.74510309,  0.58506717,\n        -1.99545251,  0.62591105,  1.80596103, -0.22309315, -1.40442136,\n        -2.1319153 ],\n       [-0.46089119, -1.43944954, -1.21809509,  0.71093011,  0.45216919,\n        -0.35953381,  0.62284954, -0.67005297, -0.7069138 ,  0.06311351,\n        -1.23199074, -1.74645233, -2.71960897, -2.21437178, -1.69307505,\n        -0.90927394,  0.87852311,  1.18664814, -1.87041262,  0.39796295,\n         1.72113872, -1.36934055,  0.8580668 , -0.24779579,  1.28009118,\n        -1.32752042]])"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta2 = np.array(weights[\"Theta2\"], dtype=\"float64\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.178930Z",
     "start_time": "2024-03-04T09:28:52.172073Z"
    }
   },
   "id": "5db486526f632a72",
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "\n",
    "data1 = loadmat(\"ex3data1.mat\")\n",
    "x = np.array(data1[\"X\"]).T\n",
    "y = np.array(data1[\"y\"], dtype=\"int8\").flatten()\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.224218Z",
     "start_time": "2024-03-04T09:28:52.178930Z"
    }
   },
   "id": "8dc55e6e91aeafdc",
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x shape = (400, 5000)\n",
      "y shape = (5000,)\n",
      "theta1 shape = (25, 401)\n",
      "theta2 shape = (10, 26)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"x shape = {}\".format(x.shape))\n",
    "print(\"y shape = {}\".format(y.shape))\n",
    "print(\"theta1 shape = {}\".format(theta1.shape))\n",
    "print(\"theta2 shape = {}\".format(theta2.shape))\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.227850Z",
     "start_time": "2024-03-04T09:28:52.224218Z"
    }
   },
   "id": "960fa58a55e04238",
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "\n",
    "def sigmoid(x):\n",
    "    return 1.0 / (1.0 + np.exp(-x))\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.234636Z",
     "start_time": "2024-03-04T09:28:52.227850Z"
    }
   },
   "id": "666a7a7905be6a22",
   "execution_count": 13
  },
  {
   "cell_type": "markdown",
   "source": [
    "\n",
    "### Network Structure\n",
    "![](./img/network.png)\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "e43e6269e04b5f57"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.9752\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def testAccuracy(theta1, theta2, x, y):\n",
    "    const_theta1 = theta1[:, 0].reshape(25, 1)\n",
    "    param_theta1 = theta1[:, 1:]\n",
    "    const_theta2 = theta2[:, 0].reshape(10, 1)\n",
    "    param_theta2 = theta2[:, 1:]\n",
    "\n",
    "    layer1 = sigmoid(param_theta1 @ x + const_theta1)\n",
    "    output_layer = sigmoid(param_theta2 @ layer1 + const_theta2)\n",
    "\n",
    "    res = np.argmax(output_layer, axis=0)\n",
    "    res = res + 1\n",
    "\n",
    "    return np.mean(res == y)\n",
    "\n",
    "\n",
    "cnt = testAccuracy(theta1, theta2, x, y)\n",
    "print(\"Accuracy = {}\".format(cnt))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:34:02.033097Z",
     "start_time": "2024-03-04T09:34:02.025742Z"
    }
   },
   "id": "3fdc3d8f49205b6d",
   "execution_count": 22
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-04T09:28:52.593228Z",
     "start_time": "2024-03-04T09:28:52.593228Z"
    }
   },
   "id": "fb4760d0b581bc8b",
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
